#!/bin/ksh
# IBM_PROLOG_BEGIN_TAG 
# This is an automatically generated prolog. 
#  
#  
#  
# Licensed Materials - Property of IBM 
#  
# (C) COPYRIGHT International Business Machines Corp. 2000,2004 
# All Rights Reserved 
#  
# US Government Users Restricted Rights - Use, duplication or 
# disclosure restricted by GSA ADP Schedule Contract with IBM Corp. 
#  
# IBM_PROLOG_END_TAG 
###################################################################
#
# Description:
#     The ctsnap command gathers configuration, log and trace information
#     about the Reliable Scalable Cluster Technology (RSCT) components that
#     are installed with BOS LINUX or AIX. This command only collects
#     the data on one local node on which this command is running.
#
#     The RSCT components include the following:
#
#       - Resource Monitoring and Control subsystem (RMC)
#
#       All IBM.* resource managers, such as:
#
#         - Audit Log resource manager (IBM.AuditRM)
#
#         - Event Response resource manager (IBM.ERRM)
#
#         - File System resource manager (IBM.FSRM)
#
#         - Host resource manager (IBM.HostRM)
#
#         - DMS resource manager (IBM.DMSRM)
#
#         - Sensor resource manager (IBM.SensorRM)
#
#       - High Availability Topology Services(HATS)
#
#       - High Availability Group Services(HAGS)
#
#     This command is typically executed when a problem is encountered with
#     any of these components in order to provide service information to the
#     IBM Support Center.
#
# Syntax:
#       ctsnap [-h] [-d output_directory]
#
# Flags:
#       -h writes the usage of this command to standard output
#       -d is used to specify the output directory (absolute path)
#          default is /tmp/ctsupt
#
# Ouputs:
#     tar file : ctsnap.host_name.nnnnnnnn.tar.Z 
#     log file : ctsnap.host_name.nnnnnnnn.log
#     (where host_name is the base name of the host name of the
#     node on which the command is running; nnnnnnnn is a timestamp).
#     By default these files are placed in the /tmp/ctsupt directory.
#
# Exit Status:
#     0 - run successfully
#     1 - error occured
#
# Security:
#     This command can only be run by a root user.
#
# Contents of the information collected:
# (LINUX or AIX in the parenthesis means now
# it is collected only on the node with LINUX
# or AIX since it is not available yet on
# another platform)
#
# 1. Files:
#
#   All files in /var/ct/
#   All files in /var/adm/ffdc/
#   All files in /usr/sbin/rsct/trctmplts/
#
#   Collect the following daemons if the
#   corresponding core files exist, otherwise
#   collect what strings on AIX or
#   strings xxx|grep "@(#)" on LINUX:
#     /usr/sbin/rsct/bin/rmcd
#     /usr/sbin/rsct/bin/IBM.*d, such as:
#       /usr/sbin/rsct/bin/IBM.SensorRMd
#       /usr/sbin/rsct/bin/IBM.Sampled
#       /usr/sbin/rsct/bin/IBM.LANRMd
#       /usr/sbin/rsct/bin/IBM.HostRMd
#       /usr/sbin/rsct/bin/IBM.FSrmd
#       /usr/sbin/rsct/bin/IBM.ERrmd
#       /usr/sbin/rsct/bin/IBM.DMSRMd
#       /usr/sbin/rsct/bin/IBM.AuditRMd
#     /usr/sbin/rsct/bin/hagsd
#     /usr/sbin/rsct/bin/hatsd, hats_nim
#
#   /etc/resol.conf
#   /etc/services
#   /var/log/messages* (LINUX)
#
# 2. Output files of commands:
#
#   errpt -a (AIX)
#
#   If a RSCT subsystem is running collect
#   lssrc -ls subsys_name, such as:
#     lssrc -ls  ctrmc
#     lssrc -ls  IBM.AuditRM
#     lssrc -ls  IBM.ERRM
#     lssrc -ls  IBM.FSRM
#     lssrc -ls  IBM.DMSRM
#     lssrc -ls  IBM.HostRM
#     lssrc -ls  IBM.SensorRM
#     and similarly to other IBM.*
#     resource managers
#     lssrc -l -s cthats
#     lssrc -l -s cthags
#   Persistent and dynamic attributes of configRM classes:     
#     lsrsrc -a b -t IBM.SharedResourceCluster
#     lsrsrc -a b -t IBM.PeerNode
#     lsrsrc -a b -t IBM.NetworkInterface
#     lsrsrc -a b -t IBM.CommunicationGroup
#     lsrsrc -a b -t IBM.RSCTParameters
#
#   ct_hats_info          and the command return code
#   ct_topology_info      and the command return code
#   ct_read_machines_lst  and the command return code
#
#   ct_hags_info  and the command return code
#   hagsns -c -s cthags
#   hagsvote -l -c -s cthags
#   hagspbs -s cthags
#   hagsgr -s cthags
#   hagsmg -s cthags
#   hagscl -l -s cthags
#   hagscounts -s cthags
#
#   ct_clusterinfo   and the command return code
#   lsclcfg          and the command return code
#
#   For network information:
#
#   ifconfig -a
#   netstat
#   netstat -r
#   netstat -i
#   netstat -rn
#   netstat -in
#   netstat -s
#   netstat -D(AIX)
#   netstat -m(AIX)
#   no -a(AIX)
#   (for LINUX, we collect all files in
#   /proc/sys/net/ipv4/conf/ and
#   /proc/sys/dev/ )
#
#   Miscellaneous command:
#
#   df -k /tmp
#   df -k /var/ct
#
#   ps -edfl|grep -E -e "IBM|rmc|hats|hags"
#
#   lssrc -a | grep -E -e "rsct|hats|hags"
#
#
#   lslpp -l "rsct.*" (AIX)
#   rpm -q rsct.core rsct.core.utils rsct.basic rsct.sdk src (LINUX)
#
#   vmstat 1 3
#
#   vmstat -s(AIX) (on linux vmstat already
#   shows info what the vmstat -s on AIX)
#
#
####################################################################
#sccsid="@(#)67   1.21.1.4   src/rsct/utils/ctsnap.sh, common_utils, rsct_rzauh, rzauh0438a 9/12/04 21:39:31"

VERSION=1.21.1.4

PATH=/usr/bin:/etc:/usr/sbin:/usr/sbin/rsct/bin:/sbin:/bin:$PATH

export LANG=C
export LC_MESSAGES=C
export LC_TIME=C

ROOTDIR=/
LOGDIR=/tmp/ctsupt
total_bytes=0

SUBSYSGRP=rsct_rm       # group name for all the resouce manager subsystems
RSCTGRP=rsct
HATSGRP=cthats
HAGSGRP=cthags

RMC=ctrmc           # rmc subsystem
HATS=cthats
HAGS=cthags

RSCTBINDIR=/usr/sbin/rsct/bin
CTDIR=/var/ct

#####################################################################
# FUNCTIONs
#####################################################################

usage () {
  echo "Usage: ctsnap [-h] [-d output_directory]"
  echo "\t-h Display this usage statement."
  echo "\t-d Directory to put information (default : /tmp/ctsupt)."
} # End of usage


valid_dir () {
  case "$1" in
        /*) echo "..................."
            ;;
        *)  echo "Invalid -d argument. You must specify an absolute path."
            exit 1
            ;;          # Not valid path.
  esac
} # End of valid_dir


function check_space {

  echo "\tChecking for required and free space ......." >> $LOGFILE 2>&1

  # adding an extra 20 kbytes into total number of bytes needed in order to
  # keep filesystem from being 100 percent full on systems tight on space

  total_bytes=`expr $total_bytes + 20`
  echo "\t\tRequired space : $total_bytes kbytes" >> $LOGFILE 2>&1

  free_space=`df -k ${LOGDIR} | tail -n +2 | awk '{print $3}'`
  echo "\t\tFree space : $free_space kbytes" >> $LOGFILE 2>&1

  if [ "$free_space" -lt "$total_bytes" ]
  then
    echo "\tNot enough space in ${LOGDIR}" 
    echo "\tNot enough space in ${LOGDIR}" >> $LOGFILE 2>&1
    echo "\tExiting utility" >> $LOGFILE 2>&1
    exit 1
  else
    echo "\tSpace checking is done" >> $LOGFILE 2>&1
  fi

} # End of check_space

function check_and_tar_symbolic_dirs {
  i=0
  circular=0
  fulldir=$new_name
  pre_dir=$new_name

  while [[ -L $fulldir ]]
    do
      if (( $i > $max_link_count )) # after max_link_count stop searching
      then
          echo "The symbolic link count exceeds max_link_count:$max_link_count." >> $LOGFILE 
          break 1
      fi
      #get the real directory name to tar
      fulldir=$(ls -ld $fulldir |awk '{print $NF}')

      # check if it is a relative dir and get the absolute path without ..
      # so that it is easy for comparison for checking circular link
      if (( $i > 0 ))
      then
          ((ii=$i-1))
          pre_dir=${dir_array[$ii]}
      fi
      if [[ $fulldir = ..* ]]
      then
          abdir=${pre_dir%/*}
          while [[ $fulldir = ..* ]]
          do
            fulldir=${fulldir#../}
            abdir=${abdir%/*}
          done
          fulldir=$abdir/$fulldir
      elif [[ $fulldir = .* ]]
      then
          fulldir=${fulldir#.}
          abdir=${pre_dir%/*}
          fulldir=$abdir$fulldir
      elif [[ $fulldir != /* ]]
      then
          abdir=${pre_dir%/*}
          fulldir=$abdir/$fulldir
      fi
      dir_array[$i]=$fulldir

      # check circular link since now they are all absolute paths
      j=0
      while (( $j < $i )) 
        do
          if [[ ${dir_array[$i]} = ${dir_array[$j]} ]]  
          then
              echo "${dir_array[$i]} is a circular link." >> $LOGFILE 
              circular=1
              break 2
          fi
          ((j=$j+1))
      done
      flist=".$fulldir $flist" # keep the original middle dirs structure
                                   # so that we can trace the links
      ((i=$i+1))
  done
  if [[ $circular != 1 ]] 
  then 
      if [[ -e ${TARFILE} ]]
      then
          tar -uf ${TARFILE} .$fulldir 2>/dev/null 
      else
          tar -cf ${TARFILE} .$fulldir 2>/dev/null 
      fi
  fi
} # End of check_and_tar_symbolic_dirs

function timeout_cmd {
    typeset sleepTime=$1 # sleep for $1 secs
    typeset timeOut=$2   # timeout in $2 secs
    typeset waitTime=0   # increment value
    typeset pid=""

    # run the cmd
    eval $cmd &
    pid=$!

    while [[ $waitTime -lt $timeOut ]]
    do
      waitTime=$(($waitTime + $sleepTime))

      if ! kill -0 $pid 2>/dev/null
      then
        isTimeOut=0
        break
      fi
      sleep $sleepTime
    done
    if [[ $waitTime -ge $timeOut ]]
    then
        kill $pid 2>/dev/null
        isTimeOut=1
        print -- "command $cmd Timed out!" >> $LOGFILE
    fi
    
} # end of timeout_cmd

function run_cmd {
  if [ "$passno" = 1 ]   # calculate total_bytes
  then
      if [ "$verbal" = 1 ]
      then
          echo "pass 1: $cmd is running.\n"
      fi
  # Section to convert commands in form of "xxxx xxx >> xxxx"
    echo "$cmd" | fgrep " >> " >/dev/null 2>&1
    if [ "$?" = 0 ]
    then
      exec 2>/dev/null
      new_cmd=`echo $cmd | sed 's/\(.*\)>>.*/\1/'`
      new_cmd="$new_cmd | wc -c"
      temp_bytes=`eval $new_cmd` 
      temp_bytes=`expr \( $temp_bytes + 512 \) \/ 1024`
      #temp_bytes=`expr $temp_bytes / 1024`
      total_bytes=`expr $total_bytes + $temp_bytes`
      exec 2>&1
      echo ".\c"
    else
  # Section to convert commands in form of "xxxx xxx > xxxx"
    echo "$cmd" | fgrep " > " >/dev/null 2>&1
      if [ "$?" = 0 ]
      then
        isTimeOut=0
        echo "$cmd" | egrep "lsrsrc|lsrpnode|lscomg" >/dev/null 2>/dev/null
        if [ "$?" = 0 ]
        then
            timeout_cmd 5 30
        fi
        if [ "$isTimeOut" = 1 ]
        then
            temp_bytes=0
        else
            exec 2>/dev/null
            new_cmd=`echo $cmd | sed 's/\(.*\)>.*/\1/'`
            new_cmd="$new_cmd | wc -c"
            temp_bytes=`eval $new_cmd` 
        fi
        temp_bytes=`expr \( $temp_bytes + 512 \) \/ 1024`
        #temp_bytes=`expr $temp_bytes / 1024`
        total_bytes=`expr $total_bytes + $temp_bytes`
        exec 2>/dev/null
        echo ".\c"
      else
  # Section to convert "ls" type commands
      echo "$cmd" | fgrep  -w "ls" >/dev/null 2>&1
        if [ "$?" = 0 ]
        then
          exec 2>/dev/null
          set $cmd
          path_name=$2
          temp_bytes=`du -s -k $path_name | awk '{print $1}'`
          total_bytes=`expr $total_bytes + $temp_bytes`
          echo ".\c"
          new_name=$path_name      
          if [[ -L $new_name ]] 
          then
              check_and_tar_symbolic_dirs #handle symbolic link
          fi 
          flist=".$new_name $flist"  # keep the original dir structure but
                                    # if it is a symbolic link then it's 
                                    # only a symbolic link when tarred
                                    # so there is no duplication
                                    #  If it's not a symbolic link then
                                    # get the file list for archive
          exec 2>&1
        else
        # for the command of form "xxx  filename.out" without ">" or ">>", 
        # but must end with ".out"
        echo "$cmd" | grep  .out >/dev/null 2>&1
          if [ "$?" = 0 ]
          then
            exec 2>/dev/null
            set $cmd
            path_name=$2
            eval $cmd
            temp_bytes=`du -s -k $path_name | awk '{print $1}'`
            total_bytes=`expr $total_bytes + $temp_bytes`
            echo ".\c"
            #don't collect twice, so delete these two lines
            #new_name=$path_name
            #flist="$new_name $flist"  #get the file list for archive
            exec 2>&1
          else
            if [ "$cmd" != "" ]
            then
              echo ". failed."
              echo "\tInvalid operation in function \"run_cmd\": " 
              echo "\t\tcmd='$cmd'" 
              echo "\t\tcomp='$comp'" 
              #print the info into dump file
              echo "\tInvalid operation in function \"run_cmd\": " >> $LOGFILE 2>&1
              echo "\t\tcmd='$cmd'" >> $LOGFILE 2>&1
              echo "\t\tcomp='$comp'" >> $LOGFILE 2>&1
              echo "\tExiting utility" >> $LOGFILE 2>&1
              exit 1
            fi
          fi
        fi
      fi
    fi
  else
    exec 2>/dev/null
    if [ "$verbal" = 1 ]
    then
        echo "command $cmd is running.\n"
    fi
    echo "$cmd" | egrep "lsrsrc|lsrpnode|lscomg"  >/dev/null 2>/dev/null 
    if [ "$?" = 0 ]
    then
        timeout_cmd 5 30
    else
        eval $cmd
    fi
    echo ".\c"
    exec 2>/dev/null
  fi
} # End of run_cmd


##############################################################################
# Function: doit
#
# To utilize this facility and add new function to collect more data from a
# system you must use one of three methods :
#
# 1) single redirection in the form of " > "
#     Notice the spaces surrounding the greater than sign
# 2) append redirection in the form of "  >> "
#     Notice the spaces surrounding the greater than signs
# 3) The ls command in the form of "ls xxxx"
#     Notice the xxxx can be either a file name or a directory name
#     (but no wildcard)
#
#   All of these forms are very important for supporting conversion of commands
#   to forms that will pipe their output to the "wc -c" command to count the
#   number of bytes for the particular operation.
#   If the commands are not in this form, you will get an error message.
#   Again, this is important to understand when adding new functionality into
#   the script for collecting more documentation
#
#   For the first (with ">") and the second (with ">>") methods you must use 
#   the following 3-line sequence to perform the appropriate space checking.
#
#   for example, for collecting "errpt -a" information :
#           comp=errpt
#           cmd="errpt -a > ./errpt.out 2>&1"
#           run_cmd
#    
#    comp    => This is the component name that will be used as the output
#               file name. This output file will be save in log directory.
#               There is no need to specify the component for the ls command.
#    cmd     => This is the actual command you would like to run in one of the
#               three appropriate formats as noted above.  Notice the " > " in
#               this form of the command.
#    run_cmd => This is just the function call to take appropriate action
#               depending on which pass we are on.
#
#   For the third (ls) method, there is no need to specify the component name,
#   only cmd and run_cmd lines are needed.
#
##############################################################################
function doit {


  #1. collect files
  #-------------------
  if [ "$passno" = 1 ]   # calculate total_bytes
  then
    #handle the files in /var/ct
    #-------------------------
    # check if configRM trace file exists, if it does convert it to a text file 
    # and put in that directory                                                 
    if [[ -f /var/ct/IW/log/mc/IBM.ConfigRM/trace ]]                            
    then                                                                        
        /usr/sbin/rsct/bin/rpttr -odtic /var/ct/IW/log/mc/IBM.ConfigRM/trace > /var/ct/IW/log/mc/IBM.ConfigRM/trace.txt
    fi                                                                          
    cmd="ls /var/ct >/dev/null 2>&1"
    run_cmd

    #handle the files in /var/adm/ffdc/
    #-------------------------
    cmd="ls /var/adm/ffdc >/dev/null 2>&1"
    run_cmd

    #handle the template files
    #-------------------------
    cmd="ls /usr/sbin/rsct/trctmplts >/dev/null 2>&1"
    run_cmd
  fi

  #check if RMC, HATS, HAGS configured
  #------------------------------------
  export LC_ALL=C
  lssrc -s $RMC >/dev/null 2>&1
  if [ "$?" = 0 ]
  then
    IF_RMC_CONFIGED="yes"
    #check if in /var/ct/<cluster>/run/mc/ core files exist
    #or to check if rmgr core files exist, as long as there is one
    #exists collect rmc and rmgr daemons: rmcd, IBM.*d
    #---------------------------------------------------
    #cluster_name=`ct_clusterinfo |grep CLUSTER_NAME | awk '{print $2}'`
    cluster_name=`lsclcfg |tail -n +2 | awk '{print $1}'`
    RMCRUNDIR=$CTDIR/$cluster_name/run
    search_dir_list="$RMCRUNDIR/mc" 
    if [[ -d $search_dir_list ]]
    then
        corefile=`find $search_dir_list -name 'core*' -print` 
        if [ "$corefile" != "" ]
          then
            HAS_RMC_CORE="yes"
        fi
    fi
    if [ "$HAS_RMC_CORE" = "no" ]
    then
      daemon_list=`ls $RSCTBINDIR/IBM*`
      tmp_daemon_list=""
      for i in $daemon_list
      do
        tmp=`basename $i`
        tmp_daemon_list="$tmp $tmp_daemon_list"
      done
      daemon_list="rmcd $tmp_daemon_list"
      if [ "$BASEOS" = "AIX" ]
      then
        for i in $daemon_list
        do
          comp=what$i
          cmd="what \$RSCTBINDIR/\$i > .${OUTDIR}/$comp.out"
          run_cmd
        done
      elif [ "$BASEOS" = "Linux" ]
      then
        for i in $daemon_list
        do
          comp=strings$i
          cmd="strings \$RSCTBINDIR/\$i|grep \"@(#)\" > .${OUTDIR}/$comp.out"
          run_cmd
        done
      fi
    else #collect rmc, rmgr daemons
      daemon_list=`ls $RSCTBINDIR/IBM*`
      daemon_list="$RSCTBINDIR/rmcd $daemon_list"
      if [ "$passno" = 1 ]   # calculate total_bytes
      then
        for i in $daemon_list
        do
          cmd="ls $i >/dev/null 2>&1"
          run_cmd
        done
      fi
    fi
  fi

  lssrc -s $HATS >/dev/null 2>&1
  if [ "$?" = 0 ]
  then
    IF_HATS_HAGS_CONFIGED="yes"
    #check if in /var/ct/<cluster>/run/cthats/ core files exist
    #
    if [[ -d $RMCRUNDIR/$HATS ]]
    then
        corefile=`find $RMCRUNDIR/$HATS -name 'core*' -print` 
        if [ "$corefile" != "" ]
        then
          HAS_HATS_CORE="yes"
        fi
    fi
    #check if in /var/ct/<cluster>/run/cthags/ core files exist
    if [[ -d $RMCRUNDIR/$HAGS ]]
    then
        corefile=`find $RMCRUNDIR/$HAGS -name 'core*' -print` 
        if [ "$corefile" != "" ]
        then
          HAS_HAGS_CORE="yes"
        fi
    fi

    if [ "$HAS_HATS_CORE" = "yes" ]
    then
      daemon_list="$RSCTBINDIR/hatsd $RSCTBINDIR/hats_nim"
      what_list="hagsd"
      if [ "$HAS_HAGS_CORE" = "yes" ]
      then
        daemon_list="$RSCTBINDIR/hagsd $daemon_list"
        what_list=""
      fi
    elif [ "$HAS_HAGS_CORE" = "yes" ]
    then
      daemon_list="$RSCTBINDIR/hagsd"
      what_list="hatsd hats_nim"
    else
      daemon_list=""
      what_list="hagsd hatsd hats_nim"
    fi

    if [ "$what_list" != "" ]
    then
      if [ "$BASEOS" = "AIX" ]
      then
        for i in $what_list
        do
          comp=what$i
          cmd="what \$RSCTBINDIR/\$i > .${OUTDIR}/$comp.out"
          run_cmd
        done
      elif [ "$BASEOS" = "Linux" ]
      then
        for i in $what_list
        do
          comp=strings$i
          cmd="strings \$RSCTBINDIR/\$i|grep \"@(#)\" > .${OUTDIR}/$comp.out"
          run_cmd
        done
      fi
    fi
    #collect hats or hags or both daemons
    if [ "$daemon_list" != "" ]
    then
      if [ "$passno" = 1 ]   # calculate total_bytes
      then
        for i in $daemon_list
        do
          cmd="ls $i >/dev/null 2>&1"
          run_cmd
        done
      fi
    fi
  fi

  if [ "$passno" = 1 ]   # calculate total_bytes
  then
    # get /etc/resolv.conf
    cmd="ls /etc/resolv.conf >/dev/null 2>&1"
    run_cmd

    #get /etc/services
    cmd="ls /etc/services >/dev/null 2>&1"
    run_cmd

    #get /etc/hosts
    cmd="ls /etc/hosts >/dev/null 2>&1"
    run_cmd

    #get /etc/inittab
    cmd="ls /etc/inittab >/dev/null 2>&1"
    run_cmd

    if [[ $ISSERVER = 1 ]]
    then
      new_name=`ls /var/log`
      for i in $new_name
      do
        cmd="ls /var/log/$i >/dev/null 2>&1"
        run_cmd
      done
    elif [ "$BASEOS" = "Linux" ]
    #get /var/log/messages*(LINUX)
    then
      new_name=`ls /var/log/messages*`
      for i in $new_name
      do
        cmd="ls $i >/dev/null 2>&1"
        run_cmd
      done
    fi

  fi

  #2. output of commands:
  #-------------------------------
  #get the errpt information(AIX)
  #------------------------------
  if [ "$BASEOS" = "AIX" ]
  then
    comp=errpt
    cmd="errpt -a > .${OUTDIR}/$comp.out"
    run_cmd

    comp=svmon
    cmd="svmon -P > .${OUTDIR}/$comp.out"
    run_cmd
  fi

  #If a RSCT subsystem is running collect lssrc -ls subsys_name output 
  #subsystem may have a status of inoperative;
  #in this case, no need to calculate the space and to run the command
  #-------------------------------------------------------------------
  #get name list of all the rm and rmc subsystems
  if [ "$IF_RMC_CONFIGED" = "yes" ]
  then
    sysname_list=`lssrc -g ${SUBSYSGRP} | tail -n +2 | awk '{print $1}'`
    sysname_list="$RMC $sysname_list"                      # add ctrmc
    if [ "$IF_HATS_HAGS_CONFIGED" = "yes" ]
    then
       sysname_list="$HATS $HAGS $sysname_list"
    fi 
    for i in $sysname_list
    do
      sys_status=`lssrc -s $i | tail -n +2 | awk '{print $3}'`
      if [ "$sys_status" = "inoperative" ]
      then
        echo "\tSubsystem $i is inoperative, command \"lssrc -ls $i\" is ignored" >> $LOGFILE 2>&1
      else
        comp=$i
        cmd="lssrc -ls $i > .${OUTDIR}/lssrc.$comp.out"
        run_cmd
      fi
    done

    # for defect 80535: add the collection of results of
    # "lsrsrc -a b -t class_name" for the following classes
    # IBM.SharedResourceCluster, IBM.PeerNode,
    # IBM.NetworkInterface, IBM.CommunicationGroup, IBM.RSCTParameters

    # IBM.SharedResourceCluster doesn't exist any more so delete it
    # resource_classes="IBM.SharedResourceCluster IBM.PeerNode IBM.NetworkInterface IBM.CommunicationGroup IBM.RSCTParameters"
#    resource_classes="IBM.PeerNode IBM.NetworkInterface IBM.CommunicationGroup IBM.RSCTParameters"
#    for i in $resource_classes
#    do
#        comp=$i
#        cmd="lsrsrc -a b -t $i > .${OUTDIR}/lsrsrc.$comp.out 2>&1"
#        run_cmd
#    done

  fi
 
  #HATS, HAGS command outputs
  #---------------------------
  if [ "$IF_HATS_HAGS_CONFIGED" = "yes" ]
  then
    #collect ctXXX command outputs except ct_clusterinfo which
    #will be collected as long as RMC is configured
    #----------------------------------------------------------
    ctcommand_list="ct_hats_info ct_topology_info ct_hags_info"
    for i in $ctcommand_list
    do
      #if the cmd exist
      comp=$i
      cmd="$i > .${OUTDIR}/$comp.out"
      run_cmd
      cmd=$i
      eval $cmd >/dev/null 2>&1
      echo "$i return code: $?" >> .${OUTDIR}/$comp.out
    done
    # special treatment for ct_read_machines_lst since its syntax is different
    # The output file is ".out" format 
    cluster_name=`lsclcfg |tail -n +2 | awk '{print $1}'`
    if [[ -e $CTDIR/$cluster_name/cfg/machine.lst ]]
    then
        if [ "$passno" = 1 ]   # calculate total_bytes and run
        then
          cmd="ct_read_machines_lst .${OUTDIR}/ct_read_machines_lst.out"
          run_cmd
          cmd="ct_read_machines_lst .${OUTDIR}/ct_read_machines_lst.out"
          eval $cmd  2>/dev/null
          echo "ct_read_machines_lst return code: $?" >> .${OUTDIR}/ct_read_machines_lst.out
        fi
    fi

    #collect hags command outputs
    #-----------------------------
    comp=hagsns 
    cmd="hagsns -c -s cthags > .${OUTDIR}/$comp.out"
    run_cmd

    comp=hagsvote
    cmd="hagsvote -l -c -s cthags > .${OUTDIR}/$comp.out"
    run_cmd

    comp=hagspbs 
    cmd="hagspbs -s cthags > .${OUTDIR}/$comp.out"
    run_cmd

    comp=hagsgr 
    cmd="hagsgr -s cthags > .${OUTDIR}/$comp.out"
    run_cmd

    comp=hagsmg 
    cmd="hagsmg -s cthags > .${OUTDIR}/$comp.out"
    run_cmd
    
    comp=hagscl 
    cmd="hagscl -l -s cthags > .${OUTDIR}/$comp.out"
    run_cmd

    comp=hagscounts 
    cmd="hagscounts -s cthags > .${OUTDIR}/$comp.out"
    run_cmd
  fi

  if [ "$IF_RMC_CONFIGED" = "yes" ]
  then
    comp=ct_clusterinfo 
    cmd="ct_clusterinfo > .${OUTDIR}/$comp.out"
    run_cmd
    cmd="ct_clusterinfo"
    eval $cmd >/dev/null 2>&1
    echo "ct_clusterinfo return code: $?" >> .${OUTDIR}/$comp.out

    comp=lsclcfg 
    cmd="lsclcfg > .${OUTDIR}/$comp.out"
    run_cmd
    cmd="lsclcfg"
    eval $cmd >/dev/null 2>&1
    echo "lsclcfg return code: $?" >> .${OUTDIR}/$comp.out
  fi

  # network informations:
  #-----------------------
  comp=ifconfig 
  cmd="ifconfig -a > .${OUTDIR}/$comp.out"
  run_cmd

  comp=netstat
  cmd="netstat > .${OUTDIR}/$comp.out"
  run_cmd

  comp=netstatr
  cmd="netstat -r > .${OUTDIR}/$comp.out"
  run_cmd

  comp=netstati
  cmd="netstat -i > .${OUTDIR}/$comp.out"
  run_cmd

  comp=netstatrn
  cmd="netstat -rn > .${OUTDIR}/$comp.out"
  run_cmd

  comp=netstatin
  cmd="netstat -in > .${OUTDIR}/$comp.out"
  run_cmd

  comp=netstats
  cmd="netstat -s > .${OUTDIR}/$comp.out"
  run_cmd
   
  if [ "$BASEOS" = "AIX" ]
  then 
    comp=netstatd
    cmd="netstat -D > .${OUTDIR}/$comp.out"
    run_cmd

    comp=netstatm
    cmd="netstat -m > .${OUTDIR}/$comp.out"
    run_cmd

    comp=no
    cmd="no -a > .${OUTDIR}/$comp.out"
    run_cmd
  elif [ "$BASEOS" = "Linux" ]
  then
    #collect files in /proc/sys/net/ipv4/conf/ and /proc/sys/dev/
    if [ "$passno" = 1 ]   # calculate total_bytes
    then
      new_dir=$LOGDIR/proc.sys.net.ipv4.conf
      if [[ -e $new_dir ]]
      then
      rm -rf $new_dir
      fi
      mkdir $new_dir

      cp -r /proc/sys/net/ipv4/conf $new_dir

      temp_bytes=`du -s -k $new_dir | awk '{print $1}'`
      total_bytes=$(($total_bytes + $temp_bytes))
      tmp_file_list="proc.sys.net.ipv4.conf $tmp_file_list"
      delete_list="$new_dir $delete_list"

      new_dir=$LOGDIR/proc.sys.dev
      if [[ -e $new_dir ]]
      then
        rm -rf $new_dir
      fi
      mkdir $new_dir

      cp -r /proc/sys/dev $new_dir

      temp_bytes=`du -s -k $new_dir | awk '{print $1}'`
      total_bytes=`expr $total_bytes + $temp_bytes`
      tmp_file_list="proc.sys.dev $tmp_file_list"
      delete_list="$new_dir $delete_list"
    fi
  fi

  # Miscellaneous command output
  #-----------------------------

  mkdir -p .${OUTDIR}/lsrsrc
  comp=lsrsrc_IBM.PeerDomain
  cmd="lsrsrc -t IBM.PeerDomain > .${OUTDIR}/lsrsrc/$comp.out"
  run_cmd

  CLNAME=$(/usr/sbin/rsct/bin/ct_clusterinfo -c)
  if [ $? -ne 0 || $CLNAME -ne "IW" ]
  then
      comp=lsrsrc_IBM.NetworkInterface
      export CT_MANAGEMENT_SCOPE=2 
      cmd="lsrsrc -t IBM.NetworkInterface > .${OUTDIR}/lsrsrc/$comp.out"
      run_cmd
  else
      comp=lsrsrc_IBM.NetworkInterface
      cmd="lsrsrc -t IBM.NetworkInterface > .${OUTDIR}/lsrsrc/$comp.out"
      run_cmd
  fi

  comp=lscomg
  cmd="lscomg > .${OUTDIR}/lsrsrc/$comp.out"
  run_cmd

  #lsrlist=`lsrsrc -x`
  #for i in $lsrlist
  #do
  #  comp=lsrsrc_$i
  #  if [[ $ISHMC = 1 || $i = \"IBM.PhysicalVolume\" || $i = \"IBM.ServiceEvent\" ]]
  #  then
  #    # only get static in cases where dynamic causes trouble
  #    cmd="lsrsrc $i > .${OUTDIR}/lsrsrc/$comp.out"
  #  else
  #    cmd="lsrsrc -A b $i > .${OUTDIR}/lsrsrc/$comp.out"
  #  fi
  #  run_cmd
  #done

   comp=lsrpnode_i
   cmd="lsrpnode -i > .${OUTDIR}/lsrsrc/$comp.out"
   run_cmd

#  comp=dftmp
#  cmd="df -k /tmp > .${OUTDIR}/$comp.out"
#  run_cmd

#  comp=dfct
#  cmd="df -k /var/ct > .${OUTDIR}/$comp.out"
#  run_cmd

  comp=df
  cmd="df -k  > .${OUTDIR}/$comp.out"
  run_cmd

  # print out header 
#  comp=ps
#  cmd="ps -edfl|head -1  > .${OUTDIR}/$comp.out"
#  run_cmd
#  comp=ps
#  cmd="ps -edfl|grep -E -e \"IBM|rmc|hats|hags|default_ip_nim\"  >> .${OUTDIR}/$comp.out"
#  run_cmd

  comp=ps
  cmd="ps -ef >  .${OUTDIR}/$comp.out"
  run_cmd


  # print out header 
#  comp=lssrc
# cmd="lssrc -a|head -1 > .${OUTDIR}/$comp.out"
#  run_cmd
#  comp=lssrc
#  cmd="lssrc -a|grep -E -e \"rsct|hats|hags\" >> .${OUTDIR}/$comp.out"
#  run_cmd

  comp=lssrc.a
  cmd="lssrc -a > .${OUTDIR}/$comp.out"
  run_cmd

  comp=vmstat
  cmd="vmstat 1 3 > .${OUTDIR}/$comp.out"
  run_cmd

  if [ "$BASEOS" = "AIX" ]
  then
    comp=vmstats
    cmd="vmstat -s > .${OUTDIR}/$comp.out"
    run_cmd

    comp=lslpp
#    cmd="lslpp -l Rsct.* > .${OUTDIR}/$comp.out"
    cmd="lslpp -ha > .${OUTDIR}/$comp.out"
    run_cmd
  fi

  comp=rpm
#  cmd="rpm -qi rsct.core rsct.core.utils rsct.basic rsct.sdk src > .${OUTDIR}/$comp.out"
  cmd="rpm -qa > .${OUTDIR}/$comp.out"
  run_cmd

  comp=uptime
  cmd="uptime > .${OUTDIR}/$comp.out"
  run_cmd

  comp=rmcdomainstatus
  cmd="rmcdomainstatus -s ctrmc > .${OUTDIR}/$comp.out"
  run_cmd

  if [[ $ISHMC = 1 ]]
  then
    comp=hsc
    cmd="hsc version > .${OUTDIR}/$comp.out"
    run_cmd

    comp=lslpars
    cmd="lslpars > .${OUTDIR}/$comp.out"
    run_cmd

    comp=getcimstatus
    cmd="getcimstatus > .${OUTDIR}/$comp.out"
    run_cmd
	
    comp=lspartition
    cmd="lspartition -dlpar > .${OUTDIR}/$comp.out"
    run_cmd

    comp=lsnode
    cmd="lsnode -H > .${OUTDIR}/$comp.out 2>&1"
    run_cmd

#    comp=lsrsrc
#    cmd="lsrsrc IBM.ManagedNode > .${OUTDIR}/lsrsrc/$comp.ManagedNode.out
#    run_cmd
    
  elif [[ $ISSERVER = 1 ]]
  then
    set -A hwclist `lsrsrc IBM.ManagedNode HWControlPoint | grep "HWControlPoint =" | cut -f2 -d '"'`
    set -A hwplist `lsrsrc IBM.ManagedNode PowerMethod | grep "PowerMethod =" | cut -f2 -d '"'`
    let i=0
    firstone=0
    rm tmpfile 2> /dev/null
    while [[ -n ${hwclist[$i]} ]]
    do
      echo "${hwclist[$i]} ${hwplist[$i]}" >> tmpfile
      let i=i+1
    done
    sort -u tmpfile > tmpfile2
    rm tmpfile
    {
      while read hwc hwp
      do
        comp=lshwinfo
        cmd="lshwinfo -p $hwp -c $hwc >> .${OUTDIR}/$comp.out"
        run_cmd 
      done
    } < tmpfile2
    rm tmpfile2


    comp=lsnode
    cmd="lsnode -H > .${OUTDIR}/$comp.out 2>&1"
    run_cmd

  fi


} # End of doit

#############################################
# MAIN
#############################################
logdate=`date +\%m\%d\%H\%M`

userid=`id -ru`
if [ "$userid" != 0 ]
then
  echo "Must be root user [0] to use this utility."
  exit 1
fi
verbal=0
while getopts "hvd:" flag
do
    case $flag in
    h)   usage
         exit 0
         ;;
    v)   verbal=1
         ;;
    d)   d_argument=$OPTARG
         LOGDIR=$d_argument
         valid_dir $LOGDIR
         ;;
    :)   print "You forgot to enter argument to $OPTARG"
         usage
         exit 1
         ;;
    \?)  usage
         exit 1
         ;;
    esac
done

cd $ROOTDIR
flist=""
tmp_file_list="ctsnap_out"
out_file=""
delete_list=""
max_link_count=32

mkdir -p ${LOGDIR}
BASEOS=`uname`
if [[ $BASEOS = "Linux" ]]
then
	/opt/hsc/bin/hsc version > /dev/null 2> /dev/null
	if [[ $? = 0 ]]
	then
		ISHMC=1			# this is an HMC
		PATH="$PATH:/opt/hsc/bin"
	fi
else	
	lslpp -L csm.server > /dev/null 2> /dev/null
	if [[ $? = 0 ]]
	then
    		value=`runact-api -c IBM.DmsCtrl::::isLicenseValid | cut -f5 -d':'`
    		if [[ $value = 1 ]]
		then
			ISSERVER=1
			PATH="$PATH:/opt/csm/bin"
		fi
	fi
fi
HOST=`hostname -s`
TARFILE=${LOGDIR}/ctsnap.${HOST}.${logdate}.tar
LOGFILE=${LOGDIR}/ctsnap.${HOST}.${logdate}.log
exec 3> $LOGFILE

OUTDIR=${LOGDIR}/ctsnap_out
if [[ -e $OUTDIR ]]
then
  rm -rf $OUTDIR
fi
mkdir -p ${OUTDIR}

IF_RMC_CONFIGED="no"
IF_HATS_HAGS_CONFIGED="no"
HAS_RMC_CORE="no"
HAS_HATS_CORE="no"
HAS_HAGS_CORE="no"

date >> $LOGFILE 2>&1
echo "Log directory set to ${LOGDIR}" >> $LOGFILE 2>&1
echo "Log directory set to ${LOGDIR}" 

echo "Calculating space requirement ......." >> $LOGFILE 2>&1

passno=1       # first pass to calculate total_bytes
doit

# check for free space in log directory
check_space

echo "Gathering information......" >> $LOGFILE 2>&1

passno=2       # second pass to gether the information
doit

echo "\n"

echo "Starting tar/compress process......" >> $LOGFILE 2>&1
flist="$flist"
set $flist

if [[ -e ${TARFILE} ]]
then
    tar -uf ${TARFILE}  -C $LOGDIR $tmp_file_list  -C $ROOTDIR "$@" 2>/dev/null
else
    tar -cf ${TARFILE}  -C $LOGDIR $tmp_file_list  -C $ROOTDIR "$@" 2>/dev/null 
fi

if [[ -e ${TARFILE} ]]
then
    which gzip  > /dev/null 2>&1
    if [[ "$?" = 0 ]]
    then
        gzip -f ${TARFILE}
    else
        which compress  > /dev/null 2>&1
        if [[ "$?" = 0 ]]
        then
            compress -f ${TARFILE}
        else
            echo "gzip and compress commands don't exist and the tar file ${TARFILE} is not compressed." >> $LOGFILE 2>&1
        fi
    fi
fi

GZFILE=${TARFILE}.gz
ZFILE=${TARFILE}.Z
out_file=".$OUTDIR $delete_list"
set $out_file
if [[ -e ${TARFILE} || -e ${GZFILE} || -e ${ZFILE} ]]
then
    rm -rf "$@"
fi

echo "*******done*******" >> $LOGFILE 2>&1

chmod 400 $LOGFILE 
if [[ -e ${GZFILE} ]]
then
    chmod 400 ${GZFILE}
elif [[ -e ${ZFILE} ]]
then
    chmod 400 $ZFILE
elif [[ -e ${TARFILE} ]]
then
    chmod 400 $TARFILE
else
    chmod 400 $out_file
fi
exit 0
